Table of Contents

Tools

In [10]:
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import pickle # import cpickle as pickle

def data_save(data,filename):
    f = open(filename, "wb") # *.dat
    pickle.dump(data, f)
    f.close()

def data_load(filename):
    return pickle.load(open(filename, "rb"))

dpi = 100

def plot_lr(x, value_history,step=600, name='plot_lr'):

    id_saved = int(step/10) # each 10 step

    max_step = value_history[0,-1,0]
    print('step=',step,'max_step=',max_step)

    plt.figure(figsize=[12,5])
    plt.subplot(121)
    plt.semilogx(x,value_history[:,id_saved,1],'b-')
    plt.plot(x,value_history[:,id_saved,3],'r-')
    #plt.xlim([0,10])
    plt.xlabel('learning rate')
    plt.ylabel('loss at step=%d'% step);
    plt.legend(('train','test'))

    plt.subplot(122)
    plt.semilogx(x,value_history[:,id_saved,2],'b-')
    plt.plot(x,value_history[:,id_saved,4],'r-')
    #plt.xlim([0,10])
    plt.xlabel('learning rate')
    plt.ylabel('accuracy at step=%d'%step);
    plt.legend(('train','test'),loc=0)

    # always call tight_layout before saving ;)
    plt.tight_layout()
    plt.savefig("./%s.png"%name, dpi=dpi)
    plt.savefig("./%s.pdf"%name)

def plot_lr2(x, vhs,step=600, name='plot_lr2'):

    vh = np.mean(vhs, axis=0)
    vh_min = np.min(vhs, axis=0)
    vh_max = np.max(vhs, axis=0)

    id_saved = int(step/10) # each 10 step

    max_step = vh[0,-1,0]
    print('step=',step,'max_step=',max_step)

    plt.figure(figsize=[12,5])
    plt.subplot(121)
    plt.semilogx(x,vh[:,id_saved,1],'b.-')
    plt.fill_between(x, vh_min[:,id_saved,1],vh_max[:,id_saved,1],color='b',alpha=0.1)
    plt.plot(x,vh[:,id_saved,3],'r.-')
    plt.fill_between(x, vh_min[:,id_saved,3],vh_max[:,id_saved,3],color='r',alpha=0.1)
    #plt.xlim([0,10])
    plt.xlabel('learning rate')
    plt.ylabel('loss at step=%d'% step);
    plt.legend(('train','test'))

    plt.subplot(122)
    plt.semilogx(x,vh[:,id_saved,2],'b.-')
    plt.fill_between(x, vh_min[:,id_saved,2],vh_max[:,id_saved,2],color='b',alpha=0.1)
    plt.plot(x,vh[:,id_saved,4],'r.-')
    plt.fill_between(x, vh_min[:,id_saved,4],vh_max[:,id_saved,4],color='r',alpha=0.1)
    #plt.xlim([0,10])
    plt.xlabel('learning rate')
    plt.ylabel('accuracy at step=%d'%step);
    plt.legend(('train','test'))

    # always call tight_layout before saving ;)
    plt.tight_layout()
    plt.savefig("./%s.png"%name, dpi=dpi)
    plt.savefig("./%s.pdf"%name)

def plot_bnsplr(x, lr_list, vhs,step=600, name='plot_bnsplr'):

    id_saved = int(step/10) # saved each 10 step
    fig,ax = plt.subplots(2, 2, figsize=(16, 8))
    for i in range(len(values)):
        vh = vhs[i]
        #print(vh[:,id_saved,0])
        max_step = vh[0,-1,0]
        ax[0,0].semilogx(x,vh[:,id_saved,1])
        ax[0,1].semilogx(x,vh[:,id_saved,2])
        ax[1,0].semilogx(x,vh[:,id_saved,3])
        ax[1,1].semilogx(x,vh[:,id_saved,4])

    print('step=',step,'max_step=',max_step)
    ax[0,0].set_ylabel('Train')
    ax[1,0].set_ylabel('Test')
    ax[0,0].set_title('Loss at step=%d' % step)
    ax[0,1].set_title('Accuracy at step=%d' % step)
    ax[1,0].set_xlabel('learning rate')
    ax[1,1].set_xlabel('learning rate')

    plt.legend(['eps_a=%g' % eps_a for eps_a in lr_list])

    # always call tight_layout before saving ;)
    plt.tight_layout()
    plt.savefig("./%s.png"%name, dpi=dpi)
    plt.savefig("./%s.pdf"%name)

def plot_bnsplr2(x, lr_list, values,step=600, name='plot_bnsplr2'):

    id_saved = int(step/10) # saved each 10 step
    fig,ax = plt.subplots(2, 2, figsize=(16, 8))
    for i in range(len(values)):
        vhs = values[i]
        vh = np.mean(vhs,axis=0)
        vh_min = np.min(vhs,axis=0)
        vh_max = np.max(vhs,axis=0)
        #print(vh[:,id_saved,0])
        max_step = vh[0,-1,0]
        ax[0,0].semilogx(x,vh[:,id_saved,1])
        ax[0,0].fill_between(x,vh_min[:,id_saved,1],vh_max[:,id_saved,1],alpha=0.1)
        ax[0,1].semilogx(x,vh[:,id_saved,2])
        ax[0,1].fill_between(x,vh_min[:,id_saved,2],vh_max[:,id_saved,2],alpha=0.1)
        ax[1,0].semilogx(x,vh[:,id_saved,3])
        ax[1,0].fill_between(x,vh_min[:,id_saved,3],vh_max[:,id_saved,3],alpha=0.1)
        ax[1,1].semilogx(x,vh[:,id_saved,4])
        ax[1,1].fill_between(x,vh_min[:,id_saved,4],vh_max[:,id_saved,4],alpha=0.1)

    print('step=',step,'max_step=',max_step)
    ax[0,0].set_ylabel('Train')
    ax[1,0].set_ylabel('Test')
    ax[0,0].set_title('Loss at step=%d' % step)
    ax[0,1].set_title('Accuracy at step=%d' % step)
    ax[1,0].set_xlabel('learning rate')
    ax[1,1].set_xlabel('learning rate')

    plt.legend(['eps_a=%g' % eps_a for eps_a in lr_list])

    # always call tight_layout before saving ;)
    plt.tight_layout()
    plt.savefig("./%s.png"%name, dpi=dpi)
    plt.savefig("./%s.pdf"%name)

/home/*/anaconda2/envs/py35/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters

Run -- MNIST

model1 - 2cnn+2fc

gd

In [76]:
%%writefile mnist_gdlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_mnist_gd()
    taskname = 'mnist_gdlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,1,20), max_step=600,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)

Overwriting mnist_gdlr.py
In [480]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/m1/mnist_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

plot_lr(x, np.mean(value_history, axis=0),step=590, name='./figures/mnist_m1_gdlr')
plot_lr2(x, value_history,step=590, name='./figures/mnist_m1_gdlr2')

step= 590 max_step= 590.0
step= 590 max_step= 590.0

bn

In [78]:
%%writefile mnist_bnlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_mnist_bn()
    taskname = 'mnist_bnlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=600,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)

Overwriting mnist_bnlr.py
In [55]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnlr_T%d.dat' % i)
    x = s[0]
    value_history.append(s[1])

plot_lr(x, np.mean(value_history, axis=0),step=600,name='./figures/mnist_m1_bnlr')
plot_lr2(x, value_history,step=600,name='./figures/mnist_m1_bnlr2')

step= 600 max_step= 600.0
step= 600 max_step= 600.0

bn_split

In [80]:
%%writefile mnist_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_mnist_bn_split()
    model.learning_rate_ab = 0.001 # 1.5, 1.0, 0.5, 0.1, 0.01, 0.001
    taskname = 'mnist_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=600,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)

Overwriting mnist_bnsplr.py
In [56]:
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    vh = []
    for i in range(5): # only one test
        s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        vh.append(s[1])

    #values.append(np.mean(vh, axis=0))
    values.append(vh)

step = 600  # 1-epoch 600 step

plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=600,name='./figures/mnist_m1_bnsplr')
plot_bnsplr2(x, lr_list, values,step=600,name='./figures/mnist_m1_bnsplr2')

step= 600 max_step= 600.0
step= 600 max_step= 600.0

combine plot

In [481]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/m1/mnist_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_gd = x
vh_gd = value_history


value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_bn = x
vh_bn = value_history


values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/m1_/mnist_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

x_bnsp = x
vhs_bnsp = values

step = 600  # 1-epoch 600 step
In [491]:
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])

from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

save_id = 59
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.7])
plt.xlabel('learning rate')
plt.ylabel('loss')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
plt.grid()

col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.94,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.48), loc=2, borderaxespad=0.)
plt.grid()

plt.tight_layout()
plt.savefig("./figures/mnist_m1_compare.png", dpi=150)
plt.savefig("./figures/mnist_m1_compare.pdf")
In [490]:
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.7])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
plt.grid()

col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.94,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
#plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.48), loc=2, borderaxespad=0.)
plt.grid()

plt.tight_layout()
plt.savefig("./figures/mnist_m1_compare_test.png", dpi=150)
plt.savefig("./figures/mnist_m1_compare_test.pdf")

Scaling check

In [51]:
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    for lrab in [1.0, 0.1, 0.01, 0.001]:
        for aug in [0.01,0.1,1,10]:
            tf.reset_default_graph()

            model = Model_mnist_bn_split()
            model.learning_rate_ab = lrab
            model.set_scaling(weight_aug=aug)
            taskname = 'mnist_m1_sca_bnsplr_ab%g_aug%g_T%d' % (lrab,aug,i)
            #tensorboard_dir = '/home/*/Results/%s/' % taskname
            tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
            test = Test()
            test.test_lr(model=model, dataset=dataset,
                         lr_list=np.logspace(-4,4,40), max_step=600,
                         logdir=tensorboard_dir)
            test.value_check()
            data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)

Writing ./script/mnist_sca_bnsplr.py
In [527]:
values = []
ilr = 0
plt.figure(figsize=[12,8])
for lrab in [1.0, 0.1, 0.01, 0.001]:
    ilr += 1
    #for aug in [0.1,1,10]: #[0.01,0.1,1,10]:
    for aug in [0.01,0.1,1,10]:
        vh = []
        for i in range(5):

            taskname = 'mnist_m1_sca_bnsplr_ab%g_aug%g_T%d' % (lrab,aug,i)
            datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
            s = data_load(datafile)
            x = s[0]
            x = s[0] / aug**2
            vh.append(s[1])
        plt.subplot(2,2,ilr)
        plt.semilogx(x,np.mean(vh,axis=0)[:,10,1])
        plt.xlabel('learning rate / aug^2')
        plt.ylabel('loss')
        plt.title('lrab=%g'%lrab)

#plt.legend(['aug=%g'%aug for aug in [0.1,1,10]])
plt.legend(['aug=%g'%aug for aug in [0.01,0.1,1,10]])
plt.tight_layout()
name = './figures/mnist_m1_sca_bnsplr'
plt.savefig("%s.png"%name, dpi=dpi)
plt.savefig("%s.pdf"%name)

/home/*/anaconda2/envs/py35/lib/python3.5/site-packages/matplotlib/cbook/deprecation.py:107: MatplotlibDeprecationWarning: Adding an axes using the same arguments as a previous axes currently reuses the earlier instance.  In a future version, a new instance will always be created and returned.  Meanwhile, this warning can be suppressed, and the future behavior ensured, by passing a unique label to each axes instance.
  warnings.warn(message, mplDeprecation, stacklevel=1)

Scaling check - add

In [281]:
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    for lrab in [1.0]: # , 0.1, 0.01, 0.001
        for aug in np.logspace(-2,2,17):#  [0.01,0.1,1,10]:
            tf.reset_default_graph()

            model = Model_mnist_bn_split()
            model.learning_rate_ab = lrab
            model.set_scaling(weight_aug=aug)
            taskname = 'mnist_m1_sca_bnsplr_add_ab%g_aug%g_T%d' % (lrab,aug,i)
            #tensorboard_dir = '/home/*/Results/%s/' % taskname
            tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
            test = Test()
            test.test_lr(model=model, dataset=dataset,
                         lr_list=np.logspace(-4,4,40)*aug**2, max_step=600,
                         logdir=tensorboard_dir)
            test.value_check()
            data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)
# killed -- so slowly

Overwriting ./script/mnist_sca_bnsplr.py
In [477]:
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    for lrab in [1.0]: # , 0.1, 0.01, 0.001
        for aug in np.logspace(-2,2,9):#  [0.01,0.1,1,10]:
            tf.reset_default_graph()

            model = Model_mnist_bn_split()
            model.learning_rate_ab = lrab
            model.set_scaling(weight_aug=aug)
            taskname = 'mnist_m1_sca_bnsplr_add2_ab%g_aug%g_T%d' % (lrab,aug,i)
            #tensorboard_dir = '/home/*/Results/%s/' % taskname
            tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
            test = Test()
            test.test_lr(model=model, dataset=dataset,
                         lr_list=np.logspace(-3,3,20)*aug**2, max_step=600,
                         logdir=tensorboard_dir)
            test.value_check()
            data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)

Overwriting ./script/mnist_sca_bnsplr.py
In [526]:
%%writefile ./script/mnist_sca_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
dataset = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    for lrab in [1.0]: # , 0.1, 0.01, 0.001
        for aug in np.logspace(-2,2,9):#  [0.01,0.1,1,10]:
            tf.reset_default_graph()

            model = Model_mnist_bn_split()
            model.learning_rate_ab = lrab
            model.set_scaling(weight_aug=aug)
            taskname = 'mnist_m1_sca_bnsplr_add3_ab%g_aug%g_T%d' % (lrab,aug,i)
            #tensorboard_dir = '/home/*/Results/%s/' % taskname
            tensorboard_dir = '/hpctmp/*/Results-mnist-m1/%s/' % taskname
            test = Test()
            test.test_lr(model=model, dataset=dataset,
                         lr_list=np.logspace(-4,4,20)*aug**2, max_step=600,
                         logdir=tensorboard_dir)
            test.value_check()
            data_save([test.lr_list,test.value_history_np], './mnist-m1/%s.dat'%taskname)

Overwriting ./script/mnist_sca_bnsplr.py
In [587]:
values = []
ilr = 0
plt.figure(figsize=[10,4])
lrab = 1
aug_list = np.logspace(-2,2,9)
#aug_list = aug_list[0:9:2]
for aug in aug_list:
#for aug in np.logspace(-2,2,17):
    vh = []
    for i in range(4):

        taskname = 'mnist_m1_sca_bnsplr_add3_ab%g_aug%g_T%d' % (lrab,aug,i)
        datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
        s = data_load(datafile)
        x = s[0] / aug**2
        vh.append(s[1])

    plt.subplot(121)
    plt.loglog(x,np.mean(vh,axis=0)[:,60,1])
    plt.xlabel('learning rate / aug^2')
    #plt.ylim([3e-3,1])
    #plt.title('lrab=%g'%lrab)
    plt.ylabel('loss')

    plt.subplot(122)
    plt.semilogx(x,np.mean(vh,axis=0)[:,60,2])
    plt.xlabel('learning rate / aug^2')
    plt.ylim([0.9,1])
    #plt.title('lrab=%g'%lrab)
    plt.ylabel('accuracy')

plt.subplot(121)
plt.legend(['aug=%g'%aug for aug in aug_list],bbox_to_anchor=(.5, 0.97), loc=2, borderaxespad=0.)
plt.subplot(122)
plt.legend(['aug=%g'%aug for aug in aug_list],bbox_to_anchor=(.5, 0.8), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/mnist_m1_bnsp_lra1_scaling_V2.png", dpi=100)
plt.savefig("./figures/mnist_m1_bnsp_lra1_scaling_V2.pdf")

/home/*/anaconda2/envs/py35/lib/python3.5/site-packages/matplotlib/cbook/deprecation.py:107: MatplotlibDeprecationWarning: Adding an axes using the same arguments as a previous axes currently reuses the earlier instance.  In a future version, a new instance will always be created and returned.  Meanwhile, this warning can be suppressed, and the future behavior ensured, by passing a unique label to each axes instance.
  warnings.warn(message, mplDeprecation, stacklevel=1)
In [568]:
values = []
ilr = 0
plt.figure(figsize=[6,4])
lrab = 1

stepid = 60
vh = []
for i in range(5):
    aug = 1
    taskname = 'mnist_m1_sca_bnsplr_add2_ab%g_aug%g_T%d' % (lrab,aug,i)
    datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
    s = data_load(datafile)
    x = s[0] / aug**2
    vh.append(s[1])
vh0 = np.mean(vh,axis=0)[:,stepid,1]


for aug in np.logspace(-2,2,9):
    vh = []
    for i in range(5):

        taskname = 'mnist_m1_sca_bnsplr_add2_ab%g_aug%g_T%d' % (lrab,aug,i)
        datafile = '/home/*/2017/hpc/bn/mnist-m1/%s.dat'%taskname
        s = data_load(datafile)
        x = s[0] / aug**2
        vh.append(s[1])
    y = np.mean(vh,axis=0)[:,stepid,1]
    #plt.loglog(x,y/vh0)
    plt.loglog(x,y)
    plt.xlabel('learning rate / aug^2')
    plt.title('lrab=%g'%lrab)


plt.legend(['aug=%g'%aug for aug in np.logspace(-2,2,9)])
Out[568]:
<matplotlib.legend.Legend at 0x7f8df9437358>

model2 - 1fc

gd

In [77]:
%%writefile mnist_m2_gdlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model2_mnist_gd()
    taskname = 'mnist_m2_gdlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results2/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,1,20), max_step=6000, # 600
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)
print('Over')

Overwriting mnist_m2_gdlr.py
In [57]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/m2m2/mnist_m2_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

plot_lr(x, np.mean(value_history, axis=0),step=600,name='./figures/mnist_m2_gdlr')
plot_lr2(x, value_history,step=600,name='./figures/mnist_m2_gdlr2')

step= 600 max_step= 6000.0
step= 600 max_step= 6000.0

bn

In [79]:
%%writefile mnist_m2_bnlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model2_mnist_bn()
    taskname = 'mnist_m2_bnlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results2/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=6000, # 600
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)

Overwriting mnist_m2_bnlr.py
In [58]:
value_history = []
for i in range(7): # add 2 test
    s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

plot_lr(x, np.mean(value_history, axis=0),step=600,name='./figures/mnist_m2_bnlr')
plot_lr2(x, value_history,step=600,name='./figures/mnist_m2_bnlr2')

step= 600 max_step= 600.0
step= 600 max_step= 600.0

bn_split

In [81]:
%%writefile mnist_m2_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model2_mnist_bn_split()
    model.learning_rate_ab = 0.001 # 1.0, 0.1, 0.01, 0.001
    taskname = 'mnist_m2_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results2/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=600, # 6000
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], '%s.dat'%taskname)

Overwriting mnist_m2_bnsplr.py
In [59]:
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    #values.append(np.mean(value_history, axis=0))
    values.append(value_history)

step = 600  # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=600,name='./figures/mnist_m2_bnsplr')
plot_bnsplr2(x, lr_list, values,step=600,name='./figures/mnist_m2_bnsplr2')

step= 600 max_step= 600.0
step= 600 max_step= 600.0

combine plot

In [493]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/m2m2/mnist_m2_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_gd = x
vh_gd = value_history


value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_bn = x
vh_bn = value_history


values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/m2/mnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

x_bnsp = x
vhs_bnsp = values

step = 600  # 1-epoch 600 step
In [494]:
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])

from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

save_id = 60
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,1.4])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.legend(strs)
plt.grid()

col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.legend(strs)
plt.grid()

plt.tight_layout()
plt.savefig("./figures/mnist_m2_compare.png", dpi=150)
plt.savefig("./figures/mnist_m2_compare.pdf")
In [495]:
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,1.4])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.legend(strs)
plt.grid()

col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.legend(strs)
plt.grid()

plt.tight_layout()
plt.savefig("./figures/mnist_m2_compare_test.png", dpi=150)
plt.savefig("./figures/mnist_m2_compare_test.pdf")

model4 - 1fc

gd

In [214]:
%%writefile ./script/mnist_m4_gdlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model4_mnist_gd()
    taskname = 'mnist_m4_gdlr_add_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-mnist-m4/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,5,40), max_step=6000,
                 #lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './mnist-m4/%s.dat'%taskname)
print('Over')

Overwriting ./script/mnist_m4_gdlr.py
In [225]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_gdlr_add_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 6000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m4_gdlr_add')
plot_lr2(x, value_history,step=step,name='./figures/mnist_m4_gdlr_add2')

step= 6000 max_step= 6000.0
step= 6000 max_step= 6000.0

bn

In [217]:
%%writefile ./script/mnist_m4_bnlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model4_mnist_bn()
    taskname = 'mnist_m4_bnlr_add_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-mnist-m4/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,5,40), max_step=6000,
                 #lr_list=np.logspace(-3,3,40), max_step=6000, # 'mnist_m4_bnlr_T%d' % i
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './mnist-m4/%s.dat'%taskname)

Overwriting ./script/mnist_m4_bnlr.py
In [229]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnlr_add_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 6000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m4_bnlr_add')
plot_lr2(x, value_history,step=step,name='./figures/mnist_m4_bnlr_add2')

step= 6000 max_step= 6000.0
step= 6000 max_step= 6000.0

bn_split

In [234]:
%%writefile ./script/mnist_m4_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

for i in range(5):
    for lrab in [1.0, 0.1, 0.01, 0.001]:
        tf.reset_default_graph()

        model = Model4_mnist_bn_split()
        model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
        taskname = 'mnist_m4_bnsplr_add_ab%g_T%d' % (model.learning_rate_ab, i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-mnist-m4/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=mnist,
                     lr_list=np.logspace(-3,7,40), max_step=6000,
                     #lr_list=np.logspace(-3,3,40), max_step=6000, # 'mnist_m4_bnsplr_ab%g_T%d'
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './mnist-m4/%s.dat'%taskname)

Overwriting ./script/mnist_m4_bnsplr.py
In [11]:
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnsplr_add_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    #values.append(np.mean(value_history, axis=0))
    values.append(value_history)

step = 6000  # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/mnist_m4_bnsplr_add')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/mnist_m4_bnsplr_add2')

step= 6000 max_step= 6000.0
step= 6000 max_step= 6000.0

combine plot

In [9]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_gdlr_add_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_gd = x
vh_gd = value_history


value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnlr_add_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_bn = x
vh_bn = value_history


values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/mnist-m4/mnist_m4_bnsplr_add_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

x_bnsp = x
vhs_bnsp = values

step = 600  # 1-epoch 600 step
In [19]:
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])

from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

save_id = 60
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.legend(strs)
plt.grid()

col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.legend(strs)
plt.grid()

plt.tight_layout()
plt.savefig("./figures/mnist_m4_compare.png", dpi=150)
plt.savefig("./figures/mnist_m4_compare.pdf")
In [95]:
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.legend(strs)
plt.grid()

col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bnsp, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bnsp, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.legend(strs)
plt.grid()

plt.tight_layout()
plt.savefig("./figures/mnist_m4_compare_test.png", dpi=150)
plt.savefig("./figures/mnist_m4_compare_test.pdf")

Run -- fashion MNIST

model1 - 2cnn+2fc

gd

In [82]:
%%writefile fmnist_m1_gdlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_mnist_gd()
    taskname = 'fmnist_m1_gdlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-fmnist-m1/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,1,20), max_step=1200,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './fmnist-m1/%s.dat'%taskname)

Overwriting fmnist_m1_gdlr.py
In [174]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 600
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m1_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m1_gdlr2')

step= 600 max_step= 1200.0
step= 600 max_step= 1200.0

bn

In [87]:
%%writefile fmnist_m1_bnlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_mnist_bn()
    taskname = 'fmnist_m1_bnlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-fmnist-m1/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=1200,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './fmnist-m1/%s.dat'%taskname)

Overwriting fmnist_m1_bnlr.py
In [175]:
value_history = []
for i in range(4):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnlr_T%d.dat' % i)
    x = s[0]
    value_history.append(s[1])

step = 600
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m1_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m1_bnlr2')

step= 600 max_step= 1200.0
step= 600 max_step= 1200.0

bn_split

In [90]:
%%writefile fmnist_m1_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_mnist_bn_split()
    model.learning_rate_ab = 0.001 # 1.0, 0.1, 0.01, 0.001
    taskname = 'fmnist_m1_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-fmnist-m1/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=1200,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './fmnist-m1/%s.dat'%taskname)

Overwriting fmnist_m1_bnsplr.py
In [63]:
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

step = 1200  # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/fmnist_m1_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/fmnist_m1_bnsplr2')

step= 1200 max_step= 1200.0
step= 1200 max_step= 1200.0

combine plot

In [499]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_gd = x
vh_gd = value_history


value_history = []
for i in range(4):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnlr_T%d.dat' % i)
    x = s[0]
    value_history.append(s[1])

x_bn = x
vh_bn = value_history


values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/fmnist-m1/fmnist_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

vhs_bnsp = values

step = 1200  # 1-epoch 600 step
In [500]:
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])

from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

save_id = 120
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0,1.0])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)

col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.8,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/fmnist_m1_compare.png", dpi=150)
plt.savefig("./figures/fmnist_m1_compare.pdf")
In [501]:
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)

col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.83,0.94])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.grid()
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/fmnist_m1_compare_test.png", dpi=150)
plt.savefig("./figures/fmnist_m1_compare_test.pdf")

model2 - 1fc

gd

In [85]:
%%writefile fmnist_m2_gdlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model2_mnist_gd()
    taskname = 'fmnist_m2_gdlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-fmnist-m2/%s/' % taskname
    test= Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,1,20), max_step=1200,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './fmnist-m2/%s.dat'%taskname)

Overwriting fmnist_m2_gdlr.py
In [64]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 1200 # 1200
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m2_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m2_gdlr2')

step= 1200 max_step= 1200.0
step= 1200 max_step= 1200.0

bn

In [88]:
%%writefile fmnist_m2_bnlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model2_mnist_bn()
    taskname = 'fmnist_m2_bnlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-fmnist-m2/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=1200,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './fmnist-m2/%s.dat'%taskname)

Overwriting fmnist_m2_bnlr.py
In [65]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 1200 # 1200
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m2_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m2_bnlr2')

step= 1200 max_step= 1200.0
step= 1200 max_step= 1200.0

bn_split

In [91]:
%%writefile fmnist_m2_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for lrab in [1.0, 0.1, 0.01, 0.001]:
    for i in range(5):
        tf.reset_default_graph()

        model = Model2_mnist_bn_split()
        model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
        taskname = 'fmnist_m2_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-fmnist-m2/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=mnist,
                     lr_list=np.logspace(-3,3,40), max_step=1200,
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './fmnist-m2/%s.dat'%taskname)

Overwriting fmnist_m2_bnsplr.py
In [66]:
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    #values.append(np.mean(value_history, axis=0))
    values.append(value_history)


step = 1200  # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/fmnist_m2_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/fmnist_m2_bnsplr2')

step= 1200 max_step= 1200.0
step= 1200 max_step= 1200.0

combine plot

In [502]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_gd = x
vh_gd = value_history


value_history = []
for i in range(4):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnlr_T%d.dat' % i)
    x = s[0]
    value_history.append(s[1])

x_bn = x
vh_bn = value_history


values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/fmnist-m2/fmnist_m2_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

vhs_bnsp = values

step = 1200  # 1-epoch 600 step
In [503]:
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])

from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

save_id = 120
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,1.2])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)

col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,0.925])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/fmnist_m2_compare.png", dpi=150)
plt.savefig("./figures/fmnist_m2_compare.pdf")
In [504]:
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.4,1.2])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.95), loc=2, borderaxespad=0.)

col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.7,0.86])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.grid()
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/fmnist_m2_compare_test.png", dpi=150)
plt.savefig("./figures/fmnist_m2_compare_test.pdf")

model3 - 2cnn+3fc

gd

In [86]:
%%writefile fmnist_m3_gdlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model3_mnist_gd()
    taskname = 'fmnist_m3_gdlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,1,20), max_step=6000,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)

Overwriting fmnist_m3_gdlr.py
In [67]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 600 # 6000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m3_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m3_gdlr2')

step= 600 max_step= 6000.0
step= 600 max_step= 6000.0

bn

In [89]:
%%writefile fmnist_m3_bnlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model3_mnist_bn()
    taskname = 'fmnist_m3_bnlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=mnist,
                 lr_list=np.logspace(-3,3,40), max_step=12000,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)

Overwriting fmnist_m3_bnlr.py
In [68]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 12000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/fmnist_m3_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/fmnist_m3_bnlr2')

step= 12000 max_step= 12000.0
step= 12000 max_step= 12000.0
In [36]:


step= 600 max_step= 12000.0
step= 600 max_step= 12000.0

bn_split

In [92]:
%%writefile fmnist_m3_bnsplr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for lrab in [1.0, 0.1, 0.01, 0.001]:
    for i in range(5):
        tf.reset_default_graph()

        model = Model3_mnist_bn_split()
        model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
        taskname = 'fmnist_m3_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=mnist,
                     lr_list=np.logspace(-3,3,40), max_step=12000,
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)

Overwriting fmnist_m3_bnsplr.py
In [130]:
%%writefile ./script/fmnist_m3_bnsplr2.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
#mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
mnist = input_data.read_data_sets("Fashion_data", one_hot=True)

for i in range(5):
    for lrab in [1.0, 0.1, 0.01, 0.001]:
        tf.reset_default_graph()

        model = Model3_mnist_bn_split()
        model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
        taskname = 'fmnist_m3_bnsplr2_ab%g_T%d' % (model.learning_rate_ab, i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-fmnist-m3/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=mnist,
                     lr_list=np.logspace(-3,3,40), max_step=1200, # re do
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './fmnist-m3/%s.dat'%taskname)

Writing ./script/fmnist_m3_bnsplr2.py
In [79]:
values = []
lr_list = [1,0.1,0.01,0.001]
#lr_list = [1]
for lr in lr_list:
    value_history = []
    for i in range(2):
        s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnsplr2_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    #values.append(np.mean(value_history, axis=0))
    values.append(value_history)

step = 600  # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/fmnist_m3_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/fmnist_m3_bnsplr2')

step= 600 max_step= 1200.0
step= 600 max_step= 1200.0

combine plot

In [505]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_gd = x
vh_gd = value_history


value_history = []
for i in range(4):
    s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnlr_T%d.dat' % i)
    x = s[0]
    value_history.append(s[1])

x_bn = x
vh_bn = value_history


values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/fmnist-m3/fmnist_m3_bnsplr2_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

vhs_bnsp = values

step = 1200  # 1-epoch 600 step
In [506]:
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])

from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

save_id = 120
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.,0.8])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)

col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.8,1])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/fmnist_m3_compare.png", dpi=150)
plt.savefig("./figures/fmnist_m3_compare.pdf")
In [507]:
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.2,0.8])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)

col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
plt.ylim([0.8,0.93])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.grid()
plt.legend(strs)
plt.legend(strs,bbox_to_anchor=(.5, 0.45), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/fmnist_m3_compare_test.png", dpi=150)
plt.savefig("./figures/fmnist_m3_compare_test.pdf")

Run -- cifar10

model1 - 2cnn+3fc

gd

In [15]:
%%writefile ./script/cifar10_m1_gdlr.py
from ModelAndTest import *
from Cifar10 import *

dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)

for i in range(5):
    tf.reset_default_graph()

    model = Model_cifar10_gd()
    taskname = 'cifar10_m1_gdlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=dataset,
                 lr_list=np.logspace(-3,1,20), max_step=1000,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)

Overwriting ./script/cifar10_m1_gdlr.py
In [422]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_gdlr')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_gdlr2')

step= 1000 max_step= 1000.0
step= 1000 max_step= 1000.0

+normalize

In [81]:
%%writefile ./script/cifar10_m1_gdlr2.py
from ModelAndTest import *
from Cifar10 import *

dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_cifar10_gd()
    taskname = 'cifar10_m1_gdlr2_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=dataset,
                 lr_list=np.logspace(-3,1,20), max_step=1000,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)

Overwriting ./script/cifar10_m1_gdlr2.py
In [82]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_gdlr2_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_gdlr_norm')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_gdlr_norm2')

step= 1000 max_step= 1000.0
step= 1000 max_step= 1000.0

bn

In [84]:
%%writefile ./script/cifar10_m1_bnlr.py
from ModelAndTest import *
from Cifar10 import *

dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)

for i in range(5):
    tf.reset_default_graph()

    model = Model_cifar10_bn()
    taskname = 'cifar10_m1_bnlr_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=dataset,
                 lr_list=np.logspace(-3,3,40), max_step=1000,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)

Overwriting ./script/cifar10_m1_bnlr.py
In [570]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_bnlr')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_bnlr2')

step= 1000 max_step= 1000.0
step= 1000 max_step= 1000.0

+normalize

In [85]:
%%writefile ./script/cifar10_m1_bnlr2.py
from ModelAndTest import *
from Cifar10 import *

dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=True)

for i in range(5):
    tf.reset_default_graph()

    model = Model_cifar10_bn()
    taskname = 'cifar10_m1_bnlr2_T%d' % i
    #tensorboard_dir = '/home/*/Results/%s/' % taskname
    tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
    test = Test()
    test.test_lr(model=model, dataset=dataset,
                 lr_list=np.logspace(-3,3,40), max_step=1000,
                 logdir=tensorboard_dir)
    test.value_check()
    data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)

Overwriting ./script/cifar10_m1_bnlr2.py
In [571]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnlr2_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

step = 1000
plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/cifar10_m1_bnlr_norm')
plot_lr2(x, value_history,step=step,name='./figures/cifar10_m1_bnlr_norm2')

step= 1000 max_step= 1000.0
step= 1000 max_step= 1000.0

bn_split

In [86]:
%%writefile ./script/cifar10_m1_bnsplr.py
from ModelAndTest import *
from Cifar10 import *

dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)

for i in range(5):
    for lrab in [1.0, 0.1, 0.01, 0.001]:

        tf.reset_default_graph()

        model = Model_cifar10_bn_split()
        model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
        taskname = 'cifar10_m1_bnsplr_ab%g_T%d' % (model.learning_rate_ab, i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=dataset,
                     lr_list=np.logspace(-3,3,40), max_step=1000,
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)

Overwriting ./script/cifar10_m1_bnsplr.py
In [535]:
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    #values.append(np.mean(value_history, axis=0))
    values.append(value_history)

step = 1000  # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/cifar10_m1_bnsplr')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/cifar10_m1_bnsplr2')

step= 1000 max_step= 1000.0
step= 1000 max_step= 1000.0

+normalize

In [87]:
%%writefile ./script/cifar10_m1_bnsplr2.py
from ModelAndTest import *
from Cifar10 import *

dataset = Cifar10(dirpath='./cifar10_data', one_hot=True, normalize=False)

for i in range(5):
    for lrab in [1.0, 0.1, 0.01, 0.001]:

        tf.reset_default_graph()

        model = Model_cifar10_bn_split()
        model.learning_rate_ab = lrab # 1.0, 0.1, 0.01, 0.001
        taskname = 'cifar10_m1_bnsplr2_ab%g_T%d' % (model.learning_rate_ab, i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-cifar10-m1/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=dataset,
                     lr_list=np.logspace(-3,3,40), max_step=1000,
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './cifar10-m1/%s.dat'%taskname)

Overwriting ./script/cifar10_m1_bnsplr2.py
In [536]:
values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnsplr2_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    #values.append(np.mean(value_history, axis=0))
    values.append(value_history)

step = 1000  # 1-epoch 600 step
plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/cifar10_m1_bnsplr_norm')
plot_bnsplr2(x, lr_list, values,step=step,name='./figures/cifar10_m1_bnsplr_norm2')

step= 1000 max_step= 1000.0
step= 1000 max_step= 1000.0

combine plot

In [572]:
value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_gdlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_gd = x
vh_gd = value_history


value_history = []
for i in range(5):
    s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnlr_T%d.dat' % (i+0))
    x = s[0]
    value_history.append(s[1])

x_bn = x
vh_bn = value_history


values = []
lr_list = [1,0.1,0.01,0.001]
for lr in lr_list:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/cifar10-m1/cifar10_m1_bnsplr_ab%g_T%d.dat' % (lr,i))
        x = s[0]
        value_history.append(s[1])

    values.append(value_history)

vhs_bnsp = values

step = 1000  # 1-epoch 600 step
In [573]:
strs = ['gd','bn']
strs.extend(['bn lr_a=%g' %lr for lr in [1,0.1,0.01,0.001]])

from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

save_id = 100
col = 1
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.98), loc=2, borderaxespad=0.)
plt.grid()

col = 2
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy')
plt.legend(strs)
plt.grid()
plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)

plt.tight_layout()
plt.savefig("./figures/cifar10_m1_compare.png", dpi=150)
plt.savefig("./figures/cifar10_m1_compare.pdf")
In [574]:
col = 3
fig = plt.figure(figsize=[12,4])
plt.subplot(121)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0,0.1])
plt.xlabel('learning rate')
plt.ylabel('loss(test)')
plt.legend(strs)
plt.grid()

col = 4
plt.subplot(122)
plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'r-')
plt.plot(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-')
plt.plot(x_bn, np.mean(values[0],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[1],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[2],axis=0)[:,save_id,col],'--')
plt.plot(x_bn, np.mean(values[3],axis=0)[:,save_id,col],'--')
#plt.ylim([0.2,1.])
plt.xlabel('learning rate')
plt.ylabel('accuracy(test)')
plt.legend(strs)
#plt.legend(strs,bbox_to_anchor=(.5, 0.5), loc=2, borderaxespad=0.)
plt.grid()

plt.tight_layout()
plt.savefig("./figures/cifar10_m1_compare_test.png", dpi=150)
plt.savefig("./figures/cifar10_m1_compare_test.pdf")

Test dimension

MNIST - model 5 - 2fc

gd

In [176]:
%%writefile ./script/mnist_m5_gdlr.py
from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

dataset = mnist
for i in range(5):
    for hidden_size in [10,100,1000]:
        tf.reset_default_graph()

        model = Model5_mnist_gd(hidden_size=hidden_size)
        taskname = 'mnist_m5_h%d_gdlr_T%d' % (hidden_size,i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-mnist-m5/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=dataset,
                     lr_list=np.logspace(-3,6,40), max_step=6000,
                     #lr_list=np.logspace(-3,5,40), max_step=6000,
                     #lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './mnist-m5/%s.dat'%taskname)
print('Over')

Overwriting ./script/mnist_m5_gdlr.py
In [227]:
for hidden_size in [10,100,1000]:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_gdlr_T%d.dat' % (hidden_size,i+0))
        x = s[0]
        value_history.append(s[1])

    step = 6000
    print(hidden_size)
    #plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m5_gdlr')
    plot_lr2(x, value_history,step=step,name='./figures/mnist_m5_gdlr')

10
step= 6000 max_step= 6000.0
100
step= 6000 max_step= 6000.0
1000
step= 6000 max_step= 6000.0

bn

In [186]:
%%writefile ./script/mnist_m5_bnlr.py

from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

dataset = mnist
for i in range(5):
    for hidden_size in [10,100,1000]:
        tf.reset_default_graph()

        model = Model5_mnist_bn(hidden_size=hidden_size)
        taskname = 'mnist_m5_h%d_bnlr_T%d' % (hidden_size,i)
        #tensorboard_dir = '/home/*/Results/%s/' % taskname
        tensorboard_dir = '/hpctmp/*/Results-mnist-m5/%s/' % taskname
        test = Test()
        test.test_lr(model=model, dataset=dataset,
                     lr_list=np.logspace(-3,6,40), max_step=6000,
                     #lr_list=np.logspace(-3,5,40), max_step=6000,
                     #lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
                     logdir=tensorboard_dir)
        test.value_check()
        data_save([test.lr_list,test.value_history_np], './mnist-m5/%s.dat'%taskname)
print('Over')

Writing ./script/mnist_m5_bnlr.py
In [229]:
for hidden_size in [10,100,1000]:
    value_history = []
    for i in range(5):
        s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnlr_T%d.dat' % (hidden_size,i+0))
        x = s[0]
        value_history.append(s[1])

    step = 6000
    print(hidden_size)
    #plot_lr(x, np.mean(value_history, axis=0),step=step,name='./figures/mnist_m5_bnlr')
    plot_lr2(x, value_history,step=step,name='./figures/mnist_m5_bnlr')

10
step= 6000 max_step= 6000.0
100
step= 6000 max_step= 6000.0
1000
step= 6000 max_step= 6000.0

bn_split

In [188]:
%%writefile ./script/mnist_m5_bnsplr.py

from ModelAndTest import *

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)

dataset = mnist
for i in range(5):
    for lrab in [10,1,0.1]:
        for hidden_size in [10,100,1000]:
            tf.reset_default_graph()

            model = Model5_mnist_bn_split(hidden_size=hidden_size)
            model.learning_rate_ab = lrab
            taskname = 'mnist_m5_h%d_bnsplr_ab%g_T%d' % (hidden_size,lrab,i)
            #tensorboard_dir = '/home/*/Results/%s/' % taskname
            tensorboard_dir = '/hpctmp/*/Results-mnist-m5/%s/' % taskname

            test = Test()
            test.test_lr(model=model, dataset=dataset,
                         #lr_list=np.logspace(-3,8,50), max_step=6000,
                         lr_list=np.logspace(-3,7,40), max_step=3000,
                         #lr_list=np.logspace(-3,7,40), max_step=6000,
                         #lr_list=np.logspace(-3,5,40), max_step=6000,
                         #lr_list=np.logspace(-3,1,20), max_step=6000, #'mnist_m4_gdlr_T%d' % i
                         logdir=tensorboard_dir)
            test.value_check()
            data_save([test.lr_list,test.value_history_np], './mnist-m5/%s.dat'%taskname)
print('Over')

Overwriting ./script/mnist_m5_bnsplr.py
In [265]:
for hidden_size in [10,100,1000]:
    lr_list = [10,1,0.1]
    values = []
    for lr in lr_list:
        value_history = []
        for i in range(5):
            s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnsplr_ab%g_T%d.dat' % (hidden_size,lr,i))
            x = s[0]
            value_history.append(s[1])

        #values.append(np.mean(value_history, axis=0))
        values.append(value_history)

    step = 3000  # 1-epoch 600 step
    step = 600  # 1-epoch 600 step
    print(hidden_size)
    # plot_bnsplr(x, lr_list, np.mean(values,axis=1),step=step,name='./figures/mnist_m5_bnsplr')
    plot_bnsplr2(x, lr_list, values,step=step,name='./figures/mnist_m5_bnsplr')

10
step= 600 max_step= 3000.0
100
step= 600 max_step= 3000.0
1000
step= 600 max_step= 3000.0

combine plot

In [444]:
nTest = 5
values_pack = {}
for hidden_size in [10,100,1000]:

    value_history = []
    for i in range(nTest):
        s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_gdlr_T%d.dat' % (hidden_size,i+0))
        x = s[0]
        value_history.append(s[1])

    x_gd = x
    vh_gd = value_history


    value_history = []
    for i in range(nTest):
        s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnlr_T%d.dat' % (hidden_size,i+0))
        x = s[0]
        value_history.append(s[1])

    x_bn = x
    vh_bn = value_history


    values = []
    lr_list = [1,0.1,0.01,0.001]
    lr_list = [100,10,1,0.1] #[1,0.1,0.01,0.001]
    lr_list = [10,1,0.1] #[1,0.1,0.01,0.001]
    for lr in lr_list:
        value_history = []
        for i in range(nTest+5):
            s = data_load('/home/*/2017/hpc/bn/mnist-m5/mnist_m5_h%d_bnsplr_ab%g_T%d.dat' % (hidden_size,lr,i))
            x = s[0]
            value_history.append(s[1])

        values.append(value_history)

    x_bnsp = x
    vhs_bnsp = values

    step = 6000  # 1-epoch 600 step
    values_pack[str(hidden_size)] = vh_gd, vh_bn, vhs_bnsp
In [273]:
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

fig = plt.figure(figsize=[16,8])
save_id = 60 # max 301, 60 is 1-epoch
col = 1

for hidden_size in [10,100,1000]:

    vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
    minloss = min(np.mean(vhs_bnsp[0],axis=0)[:,-1,col])
#     minloss = 0

    # plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'b-')
    # plt.semilogx(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-.')
    plt.loglog(x_bnsp, np.mean(vhs_bnsp[0],axis=0)[:,save_id,col]-minloss,'--')
    plt.plot(x_bnsp, np.mean(vhs_bnsp[1],axis=0)[:,save_id,col]-minloss,'-s')
    plt.plot(x_bnsp, np.mean(vhs_bnsp[2],axis=0)[:,save_id,col]-minloss,'-o')
    # plt.plot(x_bnsp, np.mean(vhs_bnsp[3],axis=0)[:,save_id,col],'--')

    # plt.plot(x_bnsp, (np.mean(vhs_bnsp[0],axis=0)[:,save_id,col]+
    #          np.mean(vhs_bnsp[1],axis=0)[:,save_id,col]+
    #          np.mean(vhs_bnsp[2],axis=0)[:,save_id,col]+
    #          np.mean(vhs_bnsp[3],axis=0)[:,save_id,col])/4,'b--')

# plt.ylim([0,0.1])
plt.xlim([1e-3,1e7]);
# plt.ylim([0.0,0.06])
plt.xlabel('learning rate')
plt.ylabel('loss')
plt.grid()
plt.legend({'10','100','1000'})
Out[273]:
<matplotlib.legend.Legend at 0x7f6958c46a58>
In [415]:
x = np.linspace(0,2*np.pi,100)
y = np.sin(x) + np.random.random(100) * 0.8

def smooth(y, m):
    ym = y+0.0
    N = len(y)
    ys = np.zeros([N+1])
    for i in range(N):
        ys[i+1] = ys[i] + y[i]
    for i in range(N):
        wid = min([m,i,N-1-i])
        #ym[i] = sum(y[i-wid:i+wid+1]) / (2*wid+1)
        ym[i] = (ys[i+wid+1] - ys[i-wid]) / (2*wid+1)
    return ym

def smooth_vh(vh, m=10): # size = [5,40,601,5]
    vh = np.array(vh)
    n1,n2,n3,n4 = vh.shape
    #m = 10;
    for i1 in range(n1):
        for i2 in range(n2):
            for i4 in range(n4):
                y = smooth(vh[i1,i2,:,i4],m)
                #y = np.exp(smooth(np.log(vh[i1,i2,:,i4]),m))
                vh[i1,i2,:,i4] = y
    return vh

def smooth_log_vh(vh, m=10): # size = [5,40,601,5]
    vh = np.array(vh)
    n1,n2,n3,n4 = vh.shape
    #m = 10;
    for i1 in range(n1):
        for i2 in range(n2):
            for i4 in range(n4):
#                 y = smooth(vh[i1,i2,:,i4],m)
                y = np.exp(smooth(np.log(vh[i1,i2,:,i4]),m))
                vh[i1,i2,:,i4] = y
    return vh

plt.plot(x, y,'o')
plt.plot(x, smooth(y,3), 'r-', lw=2)
plt.plot(x, smooth(y,5), 'g-', lw=2)
Out[415]:
[<matplotlib.lines.Line2D at 0x7f6956b3b860>]
In [ ]:
vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
In [351]:
kk=30
plt.semilogy(smooth(np.mean(vhs_bnsp[0],axis=0)[kk,:,col],1),'r-')
# plt.plot(smooth(np.mean(vhs_bnsp[0],axis=0)[kk,:,col],3),'g-')
plt.plot(smooth(np.mean(vhs_bnsp[0],axis=0)[kk,:,col],20),'b-')
Out[351]:
[<matplotlib.lines.Line2D at 0x7f6957e04c88>]
In [527]:
vhs_pack={}
for hidden_size in [10,100,1000]:

    vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
    vhs = smooth_vh(vhs_bnsp[0],ms)
    vhs_pack[str(hidden_size)] = vhs
In [624]:
from matplotlib import rcParams
rcParams['grid.linestyle'] = '-'
rcParams['grid.color'] = 'gray'
rcParams['grid.linewidth'] = 0.1

fig = plt.figure(figsize=[6,4])
save_id = 60 # max 301, 60 is 1-epoch
col = 2

ms = 1 # smooth
for hidden_size in [10,100,1000]:

    vh_gd, vh_bn, vhs_bnsp = values_pack[str(hidden_size)]
#     vhs = smooth_vh(vhs_bnsp[0],ms)
#     vhs = smooth_log_vh(vhs_bnsp[0],ms)
    vhs = vhs_pack[str(hidden_size)]

    minloss = min(np.mean(vhs,axis=0)[:,300-ms,col])
    minloss = max(np.mean(vhs,axis=0)[:,300-ms,col])
    minloss = 0
#     minloss = 1

    # plt.semilogx(x_gd, np.mean(vh_gd,axis=0)[:,save_id,col],'b-')
    # plt.semilogx(x_bn, np.mean(vh_bn,axis=0)[:,save_id,col],'b-.')
    v = np.mean(vhs,axis=0)[:,save_id,col]-minloss
    print(v.T)
    #v = np.mean(vhs_bnsp[0],axis=0)[:,save_id,col]-minloss
#     plt.loglog(x_bnsp, 1-v,'-o')
    plt.semilogx(x_bnsp, v,'--o')
#     plt.loglog(x_bnsp, v,'--o')

# plt.ylim([0,0.1])
# plt.xlim([1e-2,1e5]); 
# plt.ylim([0.0,0.06])
plt.ylim([0.7,1.03])
plt.xlabel('learning rate')
# plt.ylabel('loss')
# plt.ylabel('accuracy')
plt.ylabel('accuracy (train)')
plt.grid()
# plt.legend(['hidden=10','hidden=100','hidden=1000'])
# plt.legend(['hidden=10','hidden=100','hidden=1000'],bbox_to_anchor=(.3, 0.9), loc=2, borderaxespad=0.)
print('save_id=',save_id,'col=',col,'ms=',ms)

plt.plot([2e0,2e3],np.array([0.,0])+0.95,'b',linewidth=4)
plt.plot([1.2e0,0.8e4],np.array([0.,0])+0.98,'r',linewidth=4)
plt.plot([7e-1,1.5e4],np.array([0.,0])+0.99,'g',linewidth=4)


# plt.plot([2e0,2e3],np.array([0.,0])+0.95,'b',linewidth=4)
# plt.plot([1e0,1e4],np.array([0.,0])+0.965,'r',linewidth=4)
# plt.plot([4e-1,4e4],np.array([0.,0])+1.0,'g',linewidth=4)


# plt.legend(['hidden=10','hidden=100','hidden=1000'],bbox_to_anchor=(.3, 0.4), loc=2, borderaxespad=0.)
plt.legend(['hidden=10','hidden=100','hidden=1000','wid=1e3','wid=1e4','wid=1e5'],bbox_to_anchor=(.3, 0.5), loc=2, borderaxespad=0.)

plt.tight_layout()
# plt.savefig("./figures/mnist_m5_compare_acc.pdf")

[0.30533333 0.316      0.35733333 0.40233333 0.486      0.55766666
 0.63933334 0.74333334 0.82033333 0.88533333 0.90933333 0.922
 0.93633333 0.93733334 0.95433334 0.95300001 0.95066667 0.95566667
 0.95633334 0.95933334 0.948      0.95466667 0.95133334 0.94733333
 0.93833333 0.93366667 0.92733334 0.897      0.88233333 0.84066667
 0.80433332 0.748      0.64       0.589      0.544      0.48666666
 0.44166666 0.45266666 0.435      0.418     ]
[0.486      0.501      0.53166666 0.61366667 0.63466667 0.72033334
 0.783      0.85833334 0.899      0.919      0.94633334 0.96533334
 0.97700001 0.98400001 0.98733334 0.98866668 0.98866668 0.98966667
 0.98933334 0.98966668 0.98800001 0.98966668 0.98633334 0.98433335
 0.98366668 0.98033334 0.97833334 0.97366668 0.96300001 0.95833334
 0.931      0.913      0.88600001 0.851      0.80833333 0.75933333
 0.76566666 0.71433333 0.69233333 0.70066667]
[0.68366667 0.702      0.71733333 0.77266666 0.796      0.85133333
 0.88633333 0.914      0.93733334 0.96133333 0.97166668 0.98400001
 0.98333334 0.98900001 0.98800001 0.99166667 0.98833334 0.99000001
 0.99466667 0.99133334 0.99300001 0.99000001 0.98833334 0.98933334
 0.99333334 0.98500001 0.98800001 0.98033334 0.98466668 0.96466668
 0.946      0.92866667 0.89633333 0.86533333 0.82433333 0.8
 0.768      0.756      0.74866666 0.74066666]
save_id= 60 col= 2 ms= 1
In [539]:


save_id= 60 col= 4 ms= 1